import time
import datetime
import re
import ssl
import urllib.request
from lxml import etree
from selenium import webdriver
from selenium.webdriver.chrome.service import Service


class Html(object):
    default_headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36'}

    ssl._create_default_https_context = ssl._create_unverified_context

    def __init__(self, url, xpath=None):
        self.url = url
        self.xpath = xpath

    def get_html_2(self, headers=default_headers):
        req = urllib.request.Request(url=self.url, headers=headers)
        res = urllib.request.urlopen(req, timeout=30)
        html = res.read()
        return html

    def get_html(self, headers=default_headers):
        chrome_options = webdriver.ChromeOptions()
        # 加上下面两行，解决报错
        chrome_options.add_argument('--no-sandbox')  # 让Chrome在root权限下跑
        chrome_options.add_argument('--disable-dev-shm-usage')
        # chrome_options.add_argument('window-size=1920x3000') #指定浏览器分辨率
        chrome_options.add_argument('--disable-gpu')  # 谷歌文档提到需要加上这个属性来规避bug
        chrome_options.add_argument('--hide-scrollbars')  # 隐藏滚动条, 应对一些特殊页面
        chrome_options.add_argument(
            'blink-settings=imagesEnabled=false')  # 不加载图片, 提升速度
        # 浏览器不提供可视化页面. linux下如果系统不支持可视化不加这条会启动失败
        chrome_options.add_argument('--headless')
        # fix:DevToolsActivePort file doesn't
        chrome_options.add_argument('--remote-debugging-port=9222')

        ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.106 Safari/537.36'
        chrome_options.add_argument('--user-agent=%s' % ua)
        chrome_options.page_load_strategy = 'eager'

        ser = Service("/home/gjh/chromedriver")
        driver = webdriver.Chrome(service=ser, options=chrome_options)
        driver.get(self.url)

        html = ""
        with open("a.html", "w") as file:
            time.sleep(8)
            html = driver.page_source
            file.write(html)
        driver.quit()
        return html

    def get_elements_by_xpath(self, num=None):
        if not self.xpath:
            print("ERROR: 未设置xpath")
            return None

        html = self.get_html()
        selector = etree.HTML(html)
        elements = selector.xpath(self.xpath)
        if num:
            elements = elements if len(elements) < num else elements[:num]
        return elements


class HtmlLinks(Html):
    def __get_raw_chapter_links(self, num=None):
        linkElements = self.get_elements_by_xpath(num)
        links = {}
        for link in linkElements:
            if link.attrib['href']:
                a_text = link.text
                if not a_text:
                    content = etree.tostring(
                        link, encoding="utf-8", method='html').decode("utf-8")
                    a_text = re.sub('<[^>]*>', '', content)
                links[link.attrib['href']] = a_text.strip() if a_text else link.attrib['href']
        return links

    def get_links(self, num=None):
        links = self.__get_raw_chapter_links(num)
        idx = self.url.index("/", self.url.index("//") + 2)
        # 抓取站点url
        siteUrl = self.url[:idx]
        # 文字根url
        subUrl = self.url[idx:]

        new_links = []
        for href, title in links.items():
            if not href or href.strip() == '':
                continue
            if href.find("javascript") > -1 or href.startswith("#"):
                continue

            # e.g. /2546/89898.html, 从根路径开始
            # print(href)
            if href.startswith('//'):
                href = 'https:' + href
            elif href[0] == '/':
                href = siteUrl + href
            # // e.g. 2546/89898.html, 从当前页面相对开始
            elif not href.startswith("http"):
                href = siteUrl + subUrl[0:subUrl.rindex('/')] + "/" + href

            new_links.append([href, title])

        new_links.reverse()
        return new_links


class HtmlContent(Html):
    def get_content(self):
        paragraphs = self.get_elements_by_xpath()
        if len(paragraphs) == 0:
            return ""
        content = etree.tostring(
            paragraphs[0], encoding="utf-8", method='html').decode("utf-8")
        # content = '\n'.join(paragraphs).strip()
        return content


def test_Html():
    h = Html("https://space.bilibili.com/47331772/dynamic",
             '//a[@class="bili-dyn-card-video"]')
    links = h.get_elements_by_xpath()
    print(links)


def test_links():
    h = HtmlLinks("https://www.zhihu.com/people/laoduo6666/posts",
                  '//*[@id="Profile-posts"]/div[2]/div/div/div/h2/span/a')
    links = h.get_links()
    print(links)


if __name__ == '__main__':
    print('>>>>>>>>>> html:\t'+datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
    test_links()
